{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b1e9fa50-d96c-4268-b4d6-2b78dc331a5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import sys\n",
    "sys.path.append('../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8ec78ee2-1909-479e-81f4-3f781f10c2f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from test_model import test\n",
    "from argparse import Namespace\n",
    "import glob\n",
    "import numpy as np\n",
    "from concurrent import futures\n",
    "import multiprocessing\n",
    "import copy\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "02f827e4-a3a4-4762-933b-f952a132df28",
   "metadata": {},
   "outputs": [],
   "source": [
    "args = {\n",
    "    'data_path': '../../Data/',\n",
    "    'pretrained_model': 'incep', #supcon', #'resnet',\n",
    "    'pretrained_path': '',\n",
    "    'window_size': 29,\n",
    "    'strided': 15,\n",
    "    'batch_size': 512,\n",
    "    'num_workers': 8,\n",
    "    'trial_id': 1,\n",
    "    'ckpt' : 50,\n",
    "    'car_model': None\n",
    "}\n",
    "\n",
    "args = Namespace(**args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "942c1c1d-1e41-4f0c-ab9a-9f52ddfad2f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../save/inception1_gamma0_lr0.001_bs256_100epochs_092722_095310',\n",
       " '../save/inception2_gamma0_lr0.001_bs256_100epochs_092722_143206',\n",
       " '../save/inception3_gamma0_lr0.001_bs256_100epochs_092722_143213',\n",
       " '../save/inception4_gamma0_lr0.001_bs256_100epochs_092722_143222',\n",
       " '../save/inception5_gamma0_lr0.001_bs256_100epochs_092722_143237']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bs = 512\n",
    "pretrained_paths = sorted(glob.glob(f'../save/inception*'))\n",
    "# pretrained_paths = sorted(glob.glob(f'../save/SupCon.resnet18*_bs{bs}*'))\n",
    "# pretrained_paths = sorted(glob.glob('../save/smallresnet18.ce?_*_lr0.001_*'))\n",
    "\n",
    "pretrained_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2d414d56-4ccf-464d-b226-46bd210a2890",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading:  SupIncepResnet\n",
      "Loading:  SupIncepResnetLoading: \n",
      "Loading:  SupIncepResnet \n",
      "SupIncepResnet\n",
      "Loading:  SupIncepResnet\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0418bf45e0c44e1c9d38aa3b05fde324",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/648 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2286880706b94697851a357c8812414a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/648 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65ed29aab48c4f108a55af862f8cef55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/648 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c266863069624785bad97e4a7da642e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/648 [00:01<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee740ae913e74ee9a5b7c5ad7236ae45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/648 [00:01<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dhkim1/miniconda3/envs/torch/lib/python3.7/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /opt/conda/conda-bld/pytorch_1623448224956/work/c10/core/TensorImpl.h:1156.)\n",
      "  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n",
      "/home/dhkim1/miniconda3/envs/torch/lib/python3.7/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /opt/conda/conda-bld/pytorch_1623448224956/work/c10/core/TensorImpl.h:1156.)\n",
      "  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n",
      "/home/dhkim1/miniconda3/envs/torch/lib/python3.7/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /opt/conda/conda-bld/pytorch_1623448224956/work/c10/core/TensorImpl.h:1156.)\n",
      "  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n",
      "/home/dhkim1/miniconda3/envs/torch/lib/python3.7/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /opt/conda/conda-bld/pytorch_1623448224956/work/c10/core/TensorImpl.h:1156.)\n",
      "  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n",
      "/home/dhkim1/miniconda3/envs/torch/lib/python3.7/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /opt/conda/conda-bld/pytorch_1623448224956/work/c10/core/TensorImpl.h:1156.)\n",
      "  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['0.0186', '0.3269', '0.2984', '0.1347', '0.1619']\n",
      "['0.9998', '0.9967', '0.9970', '0.9987', '0.9984']\n",
      "['0.9988', '1.0000', '0.9990', '0.9997', '0.9995']\n",
      "['0.9993', '0.9983', '0.9980', '0.9992', '0.9990']\n",
      "['0.0112', '0.2670', '0.3411', '0.2509', '0.1594']\n",
      "['0.9999', '0.9973', '0.9966', '0.9975', '0.9984']\n",
      "['0.9986', '1.0000', '0.9994', '0.9998', '0.9996']\n",
      "['0.9992', '0.9986', '0.9980', '0.9987', '0.9990']\n",
      "['0.0166', '0.2072', '0.3449', '0.2192', '0.1957']\n",
      "['0.9998', '0.9979', '0.9966', '0.9978', '0.9980']\n",
      "['0.9987', '0.9998', '0.9992', '0.9994', '0.9998']\n",
      "['0.9992', '0.9988', '0.9979', '0.9986', '0.9989']\n",
      "['0.0093', '0.4097', '0.2558', '0.2007', '0.1933']\n",
      "['0.9999', '0.9959', '0.9974', '0.9980', '0.9981']\n",
      "['0.9986', '1.0000', '0.9993', '0.9997', '0.9997']\n",
      "['0.9992', '0.9979', '0.9984', '0.9989', '0.9989']\n",
      "['0.0127', '0.2808', '0.4186', '0.1822', '0.2271']\n",
      "['0.9999', '0.9972', '0.9958', '0.9982', '0.9977']\n",
      "['0.9985', '1.0000', '0.9992', '0.9994', '0.9998']\n",
      "['0.9992', '0.9986', '0.9975', '0.9988', '0.9987']\n"
     ]
    }
   ],
   "source": [
    "num_cpu = multiprocessing.cpu_count()\n",
    "workers = max(num_cpu, len(pretrained_paths))\n",
    "\n",
    "with futures.ProcessPoolExecutor(workers) as executor:\n",
    "    to_do = []\n",
    "    for path in pretrained_paths:\n",
    "        args.trial_id = path.split('_')[0][-1]\n",
    "        args.pretrained_path = f'{path}/models/'\n",
    "        future = executor.submit(test, copy.deepcopy(args), \n",
    "                                 {'verbose': False, 'is_cuda': True})\n",
    "        to_do.append(future)\n",
    "        \n",
    "    \n",
    "    total_results = {}\n",
    "    for future in futures.as_completed(to_do):\n",
    "        try:\n",
    "            results = future.result()\n",
    "            for k, v in results.items():\n",
    "                total_results.setdefault(k, [])\n",
    "                total_results[k].append(v)\n",
    "        except Exception as error:\n",
    "            print('An exception occurred: {}'.format(error))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d4ccb675-670d-4818-83d7-98a9b4cb4b79",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final results\n",
      "fnr ['0.0137', '0.2983', '0.3318', '0.1976', '0.1875']\n",
      "rec ['0.9999', '0.9970', '0.9967', '0.9980', '0.9981']\n",
      "pre ['0.9986', '0.9999', '0.9992', '0.9996', '0.9997']\n",
      "f1 ['0.9992', '0.9985', '0.9979', '0.9988', '0.9989']\n"
     ]
    }
   ],
   "source": [
    "print('Final results')\n",
    "total_results = {k: np.stack(v, axis=0) for k, v in total_results.items()}\n",
    "for k, v in total_results.items():\n",
    "    print(k, list(\"{0:0.4f}\".format(i) for i in v.mean(axis=0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a629379c-1356-4130-bb51-4ba70ba6b9a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NumpyEncoder(json.JSONEncoder):\n",
    "    def default(self, obj):\n",
    "        if isinstance(obj, np.ndarray):\n",
    "            return obj.tolist()\n",
    "        return json.JSONEncoder.default(self, obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "92d467bf-3823-40ee-832f-fa9278b54f57",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "file = f'{args.pretrained_model}{bs}.json'\n",
    "with open(f'../reports/{file}', 'w') as convert_file:\n",
    "     convert_file.write(json.dumps(total_results, cls=NumpyEncoder))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.10 ('torch')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "ec8a7a313ab33d199c8aa698bb86bd912b8385ce4922a6e184e3f5edd5eb95f6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
